# # -*- coding: utf-8 -*-

# # import pandas as pd

# # # 机构职能类>>csv_file
# # file_path = r'/data/lrs/QAnything/qanything_kernel/raw_data/机构职能类_format2.xlsx'
# # df = pd.read_excel(file_path)
# # df.rename(columns={'content-by': 'content'}, inplace=True)
# # df[['content']].to_csv('/data/lrs/QAnything/qanything_kernel/data/机构职能类.csv', index=False, encoding='utf-8')


# # import pandas as pd
# # from html2text import html2text
# # import re

# # zsk_data_path = r'/data/lrs/QAnything/qanything_kernel/raw_data/all_vaild_kldata.csv'

# # df = pd.read_csv(zsk_data_path, delimiter='$')

# # # 去除 KL_CONTENT_TYPE 列中的缺失值
# # df.dropna(subset=['KL_CONTENT_TYPE'], inplace=True)

# # import os
# # import csv
# # from uuid import uuid4

# # db_dir = r'/data/yf_center/jingbo/QAnything_pro/QAnything-master/QANY_LOCAL_DB/2024-03-13'

# # def write_to_csv(title, data):
# #     # 根据 title 生成文件名
# #     if title:
# #         file_name = f"{title}.csv"
# #     else:
# #         file_name = f"{uuid4()}.csv"
# #     file_path = os.path.join(db_dir, file_name)
# #     if os.path.exists(file_path):
# #         # 如果文件已存在，则添加唯一标识符
# #         file_name = f"{title}_{uuid4()}.csv"
# #         file_path = os.path.join(db_dir, file_name)

# #     # 创建 CSV 文件并写入数据
# #     with open(file_path, 'w', newline='', encoding='utf-8') as csvfile:
# #         writer = csv.writer(csvfile)
# #         for row in data:
# #             writer.writerow(row)

# # for i in range(len(df)):
# # # for i in range(2):
# #     try:
# #         print(f'正在处理{i+1}个文件！')
# #         title = df['KL_CONTENT_TITLE'][i]
# #         content = html2text(df['KL_CONTENT'][i])
# #         # if df['KL_CONTENT_KEYWORD'][i]:
# #         #     kb_content = title + '\n' + '关键词：' + df['KL_CONTENT_KEYWORD'][i] + '\n' + content 
# #         # else:
# #         #     kb_content = title + '\n' + content
# #         kb_content = title + '\n' + content
# #         kb_content = re.sub(r'\n+', '\n', kb_content)
# #         kb_content = kb_content.rstrip('\n')
# #         write_to_csv(title, [['content'], [kb_content]])
# #     except Exception as e:
# #         print(f'{i+1}')
# #         continue




# # import pandas as pd
# # from html2text import html2text
# # import re

# # # zsk_data_path = r'/workspace/qanything_local/qanything_kernel/row_data/poc_1.7w+_data/2024-03-29_市热线-知识库点点通信息_17603.csv'
# # # zsk_data_path = r'/workspace/qanything_local/qanything_kernel/row_data/poc_1.7w+_data/2024-04-23_政务poc.csv'

# # zsk_data_path = '/workspace/qanything_local/qanything_kernel/row_data/临港算力知识库/KG_20241012_临港算力业务介绍.xlsx'


# # # df = pd.read_csv(zsk_data_path)
# # # df = pd.read_csv(zsk_data_path, encoding='utf8')  # ['KL_ID', 'KL_CONTENT_KEYWORD', 'KL_CONTENT_TITLE', 'KL_CONTENT', 'KL_CONTENT_TYPE']

# # df = pd.read_excel(zsk_data_path)

# # # 去除 '内容' 列中的缺失值
# # # df.dropna(subset=['内容'], inplace=True)
# # print(len(df))
# # df.dropna(subset=['内容'], inplace=True)

# # import os
# # import csv
# # from uuid import uuid4

# # db_dir = r'/workspace/qanything_local/QANY_LOCAL_DB/poc_zsk-2024-04-23_17K'

# # # def write_to_csv(title, data):
# # #     # 根据 title 生成文件名
# # #     if title:
# # #         file_name = f"{title}.csv"
# # #     else:
# # #         file_name = f"{uuid4()}.csv"
# # #     file_path = os.path.join(db_dir, file_name)
# # #     if os.path.exists(file_path):
# # #         # 如果文件已存在，则添加唯一标识符
# # #         file_name = f"{title}_{uuid4()}.csv"
# # #         file_path = os.path.join(db_dir, file_name)

# # #     # 创建 CSV 文件并写入数据
# # #     try:
# # #         with open(file_path, 'w', newline='', encoding='utf-8') as csvfile:
# # #             writer = csv.writer(csvfile)
# # #             for row in data:
# # #                 writer.writerow(row)
# # #     except Exception as e:
# # #         print(f'写入文件时出现错误：{e}')
# # #         file_name = f"{uuid4()}.csv"
# # #         file_path = os.path.join(db_dir, file_name)
# # #         try:
# # #             with open(file_path, 'w', newline='', encoding='utf-8') as csvfile:
# # #                 writer = csv.writer(csvfile)
# # #                 for row in data:
# # #                     writer.writerow(row)
# # #         except Exception as e:
# # #             print(f'创建新文件时出现错误：{e}')
# # #         else:
# # #             pass


# # # # for i in range(len(df)):
# # # for i in range(1776, 1777):
# # #     # print(f'正在处理{i+1}个文件！')
# # #     try:
# # #         title = df['标题'][i]
# # #         key_words = df['关键字'][i]
# # #         content = html2text(df['内容'][i])
# # #         content_category = df['内容分类'][i]
# # #         unit = df['提供单位'][i]

# # #         kb_content = '标题：' + title + '\n' + '关键字：' + key_words + '\n' + '内容：\n' + content + '\n' + '内容分类：' + content_category + '\n' + '提供单位：' + unit

# # #         kb_content = re.sub(r'\n+', '\n', kb_content)
# # #         kb_content = kb_content.rstrip('\n')
# # #         write_to_csv(title, [['content'], [kb_content]])
    
# # #     except Exception as e:
# # #         print(f'{i+1}文件存在问题,请检查')
# # #         continue

# # def truncate_title(title, max_length=25):
# #     """截断标题到指定长度，并移除不安全的文件名字符"""
# #     # 移除不安全的文件名字符
# #     safe_title = re.sub(r'[\\/*?:"<>|]', '_', title)
# #     # 截断标题到最大长度
# #     return safe_title[:max_length].strip()

# # def write_to_csv(zsk_file_id, title, key_words, content):
# #     # 确保文件名是安全的，不会与文件系统的路径分隔符冲突
# #     title = truncate_title(title)

# #     # 根据 title 生成文件名
# #     if title:
# #         file_name = f"{title}.csv"
# #     else:
# #         file_name = f"{uuid4()}.csv"
# #     file_path = os.path.join(db_dir, file_name)

# #     # 如果文件已存在，则添加唯一标识符
# #     if os.path.exists(file_path):
# #         file_name = f"{title}_{uuid4()}.csv"
# #         file_path = os.path.join(db_dir, file_name)

# #     # 创建 CSV 文件并写入数据
# #     try:
# #         with open(file_path, 'w', newline='', encoding='utf-8') as csvfile:
# #             writer = csv.writer(csvfile)
# #             # 写入标题行
# #             writer.writerow(['zsk_file_id', 'title', 'keywords', 'content'])
# #             # 写入内容行
# #             writer.writerow([zsk_file_id, title, key_words, content])
# #     except Exception as e:
# #         print(f'写入文件时出现错误：{e}')
# #         # 如果写入失败，尝试创建一个新的文件名
# #         file_name = f"{uuid.uuid4()}.csv"
# #         file_path = os.path.join(db_dir, file_name)
# #         try:
# #             with open(file_path, 'w', newline='', encoding='utf-8') as csvfile:
# #                 writer = csv.writer(csvfile)
# #                 writer.writerow(['zsk_file_id', 'title', 'keywords', 'content'])
# #                 writer.writerow([zsk_file_id, title, key_words, content])
# #         except Exception as e:
# #             print(f'创建新文件时出现错误：{e}')
# #         else:
# #             pass  

# # # 假设 df 是一个包含所需数据的 DataFrame
# # for i in range(len(df)):
# # # for i in range(1776):
# #     try:
# #         zsk_file_id = df.iloc[i]["知识库ID"]
# #         title = df.iloc[i]['标题']
# #         key_words = df.iloc[i]['关键字']
# #         content = html2text(df.iloc[i]['内容'])
# #         content_category = df.iloc[i]['内容分类']
# #         unit = df.iloc[i]['提供单位']

# #         if key_words:
# #             kb_content = '标题：' + title + '\n' + '关键字：' + key_words + '\n' + '内容：\n' + content + '\n' + '内容分类：' + content_category + '\n' + '提供单位：' + unit
# #         else:
# #             kb_content = '标题：' + title + '\n' + '内容：\n' + content + '\n' + '内容分类：' + content_category + '\n' + '提供单位：' + unit

# #         kb_content = re.sub(r'\n+', '\n', kb_content)
# #         kb_content = kb_content.rstrip('\n')
# #         write_to_csv(zsk_file_id, title, key_words, kb_content)
    
# #     except Exception as e:
# #         print(f'{i+1}文件存在问题,请检查')
# #         continue

# # print('过滤后文件数量：', len(df))
# # for i in range(len(df)):
# # # for i in range(2):  # test
# #     try:
# #         zsk_file_id = df.iloc[i]["KL_ID"]
# #         title = df.iloc[i]['KL_CONTENT_TITLE']
# #         key_words = df.iloc[i]['KL_CONTENT_KEYWORD']
# #         content = html2text(df.iloc[i]['KL_CONTENT'])
# #         content_category = df.iloc[i]['KL_CONTENT_TYPE']

# #         # kb_content = '标题：' + title + '\n' + '关键字：' + key_words + '\n' + '内容：\n' + content + '\n' + '内容分类：' + content_category + '\n' + '提供单位：' + unit
# #         kb_content = '标题：' + title + '\n' + '关键字：' + key_words + '\n' + '内容：\n' + content + '\n' + '内容分类：' + content_category

# #         kb_content = re.sub(r'\n+', '\n', kb_content)
# #         kb_content = kb_content.rstrip('\n')
# #         write_to_csv(zsk_file_id, title, key_words, kb_content)
    
# #     except Exception as e:
# #         print(f'{i+1}文件存在问题,请检查')
# #         continue



# # nohup python3 -u process_data.py > /workspace/qanything_local/logs/debug_logs/test_zhengwu_data.log 2>&1 &
    



# ######################### 临港知识库 #################################
# import re
# import os
# import csv
# from uuid import uuid4
# import pandas as pd

# zsk_data_path = '/workspace/qanything_local/qanything_kernel/row_data/临港算力知识库/KG_20241012_临港算力业务介绍.xlsx'


# df = pd.read_excel(zsk_data_path)

# print(len(df))
# df.dropna(subset=['content'], inplace=True)

# db_dir = '/workspace/qanything_local/QANY_LOCAL_DB/lingang_zsk-2024-10-12'

# def truncate_title(title, max_length=25):
#     """截断标题到指定长度，并移除不安全的文件名字符"""
#     # 移除不安全的文件名字符
#     safe_title = re.sub(r'[\\/*?:"<>|]', '_', title)
#     # 截断标题到最大长度
#     return safe_title[:max_length].strip()

# def write_to_csv(zsk_file_id, title, key_words, content):
#     # 确保文件名是安全的，不会与文件系统的路径分隔符冲突
#     title = truncate_title(title)

#     # 根据 title 生成文件名
#     if title:
#         file_name = f"{title}.csv"
#     else:
#         file_name = f"{uuid4()}.csv"
#     file_path = os.path.join(db_dir, file_name)

#     # 如果文件已存在，则添加唯一标识符
#     if os.path.exists(file_path):
#         file_name = f"{title}_{uuid4()}.csv"
#         file_path = os.path.join(db_dir, file_name)

#     # 创建 CSV 文件并写入数据
#     try:
#         with open(file_path, 'w', newline='', encoding='utf-8') as csvfile:
#             writer = csv.writer(csvfile)
#             # 写入标题行
#             writer.writerow(['zsk_file_id', 'title', 'keywords', 'content'])
#             # 写入内容行
#             writer.writerow([zsk_file_id, title, key_words, content])
#     except Exception as e:
#         print(f'写入文件时出现错误：{e}')
#         # 如果写入失败，尝试创建一个新的文件名
#         file_name = f"{uuid.uuid4()}.csv"
#         file_path = os.path.join(db_dir, file_name)
#         try:
#             with open(file_path, 'w', newline='', encoding='utf-8') as csvfile:
#                 writer = csv.writer(csvfile)
#                 writer.writerow(['zsk_file_id', 'title', 'keywords', 'content'])
#                 writer.writerow([zsk_file_id, title, key_words, content])
#         except Exception as e:
#             print(f'创建新文件时出现错误：{e}')
#         else:
#             pass  

# for i in range(len(df)):
#     try:
#         zsk_file_id = uuid4()
#         title = df.iloc[i]['title']
#         key_words = ""
#         content = df.iloc[i]['content']

#         kb_content = '标题:\n' + title + '\n' + '内容:\n' + content

#         kb_content = re.sub(r'\n+', '\n', kb_content)
#         kb_content = kb_content.rstrip('\n')
#         write_to_csv(zsk_file_id, title, key_words, kb_content)
    
#     except Exception as e:
#         print(f'{i+1}文件存在问题,请检查')
#         continue



def f(plans):
    if not plans:
        return []
    
    res = {}

    for p in plans:
        if p not in res:
            res[p] = 1

        else:
            res[p] += 1

    res = sorted(res.items(), key=lambda item: item[1], reverse=True)  # [(k, v), ...]

    s = []
    for k, v in res:
        if v >= 3:
            s.append(k)

    return s